package cn.doitedu.ml.examples

import org.apache.spark.ml.linalg.Vectors
import org.apache.spark.ml.stat.Correlation
import org.apache.spark.sql.SparkSession

import scala.collection.mutable

object CorvDemo {


  def main(args: Array[String]): Unit = {
    val spark = SparkSession.builder().master("local").appName("").getOrCreate()
    import spark.implicits._
    import org.apache.spark.sql.functions._

    // val df = spark.read.option("header", true).option("inferSchema",true).csv("portrait/testdata/cov/sample/1.csv")
    // df.select('x1,'x2,'y).show(100,false)
    // println(df.select(variance('x1) as "v").head().get(0))
    // println(df.select(covar_pop('x1,'y) as "c").head().get(0))

    //val df = spark.read.option("header", true).option("inferSchema",true).csv("portrait/testdata/cov/sample/3.csv")
    //println(df.select(covar_pop('x, 'y)).head().get(0))
    //println(df.select(covar_pop('x, 'y)).head().get(0))

    //val arr2vec = udf((arr:mutable.WrappedArray[Double])=>{
    //  Vectors.dense(arr.toArray)
    //})
    //val vec1 = df.select(arr2vec(array('x)) as "feature")
    //val vec2 = df.select(arr2vec(array('x,'y)) as "feature")
    //Correlation.corr(vec1,"feature").show(100,false)
    //Correlation.corr(vec2,"feature").show(100,false)



    val df = spark.read.option("header", true).option("inferSchema",true).csv("portrait/testdata/cov/sample/1.csv")
    val arr2vec = udf((arr:mutable.WrappedArray[Double])=>{
      Vectors.dense(arr.toArray)
    })
    val vec = df.select(arr2vec(array('x1,'x2,'y)) as "feature")
    Correlation.corr(vec,"feature").show(100,false)

    spark.close()


  }

}
